package com.bbz.crawler.core.page.processor;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.bbz.crawler.core.page.dataprocess.GithubJavaDataProcess;
import com.bbz.crawler.core.page.model.GithubJavaModel;
import com.bbz.crawler.core.page.pipeline.GithubForJavaAllPipeline;
import com.bbz.crawler.core.util.HttpClientDownloader;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

@Component
public class GithubForJavaAllProcessor implements PageProcessor {
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(10000);
	@Autowired
	GithubForJavaAllPipeline githubForJavaAllPipeline;

	@Override
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		String url = page.getHtml().xpath("//a[@class='next_page']/@href").toString();
		GithubJavaModel githubJavaModel = new GithubJavaDataProcess().dataProcess(page);
		if (githubJavaModel.getCrawlerResult() == null) {
			// skip this page
			page.setSkip(true);
		} else {
			page.putField("githubJavaModel", githubJavaModel);
		}
		// URL加到抓取列表中去
		page.addTargetRequest(url);
	}

	@Override
	public Site getSite() {
		return site;
	}

	public void run() {
		Spider.create(new GithubForJavaAllProcessor())
				// 从https://github.com/search?l=Java&p=1&q=stars%3A%3E1&s=stars&type=Repositories开始抓
				.addUrl("https://github.com/search?l=Java&p=1&q=stars%3A%3E1&s=stars&type=Repositories")
				// 添加pipeline
				.addPipeline(githubForJavaAllPipeline)
				// 开启5个线程抓取
				.thread(5).setDownloader(new HttpClientDownloader())
				.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)))
				// 启动爬虫
				.run();
	}

}
